# HTML文档内容提取器
# by lw 2023-11-7

import requests
from bs4 import BeautifulSoup
from lxml import etree
import re


class HTMLAnalyzer:

    def __init__(self):
        self.soup = None
        self.tree = None

    def load_page(self, url: str) -> bool:
        """
        从指定URL加载页面内容并初始化BeautifulSoup、etree对象

        Args:
            url (str): 页面URL

        Returns:
            bool: 加载是否成功
        """
        try:
            response = requests.get(url)
            html_content = response.content
            self.soup = BeautifulSoup(html_content, 'html.parser')
            self.tree = etree.HTML(html_content)
            return True
        except Exception as e:
            print("Failed to load page:", e)
            return False

    def load_html(self, html_content):
        """
        加载HTML内容并初始化BeautifulSoup、etree对象

        Args:
            html_content (str): HTML内容
        """
        self.soup = BeautifulSoup(html_content, 'html.parser')
        self.tree = etree.HTML(html_content)

    def xextract_keyword_content(self, keyword: str, path: str,
                                 reg: re) -> str:
        """
        使用XPath提取关键词内容

        Args:
            keyword (str): 关键词
            path (str): XPath路径
            reg (re): 匹配内容的正则表达式

        Returns:
            str: 提取的关键词内容
        """
        keyword_content = ""
        element = self.tree.xpath(path + '//text()[contains(., "' + keyword +
                                  '")]')
        if element:
            keyword_content_text = element[0]
            match = re.search(reg, keyword_content_text)
            if match:
                keyword_content = match.group(1)
        return keyword_content

    def locate_box(self, tag_desc: dict):
        """
        依据容器的定位描述搜索一个节点

        Args:
            keyword (:str): 关键词\n
            tag_desc (:dict):父容器的定位描述，采用如下键值对格式：
                                {
                                    'name':'div', # 标签的名称，可为字符串或正则表达式。\n
                                    'id':'noticeArea', # 标签的ID\n
                                    'attrs':{‘class’: ‘test’} # 标签的属性，字典类型\n
                                }

        Returns:
            any: 提取的关键词内容
        """
        tag_name = None
        tag_id = None
        tag_attrs = None

        if tag_desc and 'name' in tag_desc and tag_desc['name'] != '':
            tag_name = tag_desc['name']
        if tag_desc and 'id' in tag_desc and tag_desc['id'] != '':
            tag_id = tag_desc['id']
        if tag_desc and 'attrs' in tag_desc and tag_desc['attrs'] != '':
            tag_attrs = tag_desc['attrs']
        if tag_id:
            if tag_attrs:
                locatedbox = self.soup.find(name=tag_name,
                                            id=tag_id,
                                            attrs=tag_attrs)
            else:
                locatedbox = self.soup.find(name=tag_name, id=tag_id)
        else:
            locatedbox = self.soup.find(name=tag_name, attrs=tag_attrs)
        return locatedbox

    def find_all_subbox(self, container, part_desc: dict) -> []:
        """
        在box中，依据容器的定位描述搜索全部节点

        Args:
            container (): 父级容器
            part_desc (:dict):循环部分包裹的容器的定位描述，采用如下键值对格式：
                                {
                                    'name':'div', # 标签的名称，可为字符串或正则表达式。\n
                                    'attrs':{‘class’: ‘test’} # 标签的属性，字典类型\n
                                }

        Returns:
            []: 循环体部分全部part容器
        """
        part_name = None
        part_attrs = None

        if part_desc and 'name' in part_desc and part_desc['name'] != '':
            part_name = part_desc['name']
        if part_desc and 'attrs' in part_desc and part_desc['attrs'] != '':
            part_attrs = part_desc['attrs']
        parts = container.find_all(name=part_name,
                                   attrs=part_attrs,
                                   recursive=False)
        return parts

    def locate(self, keyword: str, parent_desc: dict = None) -> list:
        """
        依据父容器的定位描述查找包含关键词的节点

        Args:
            keyword (:str): 关键词\n
            parent_desc (:dict): 父容器的定位描述，采用如下键值对格式：
                                {
                                    'name':'div', # 标签的名称，可为字符串或正则表达式。\n
                                    'id':'noticeArea', # 标签的ID\n
                                    'attrs':{‘class’: ‘test’} # 标签的属性，字典类型\n
                                }

        Returns:
            list: 包含所有匹配节点的列表
        """
        if parent_desc:
            parent_tag = self.locate_box(tag_desc=parent_desc)
        else:
            parent_tag = self.soup
        matching_nodes = []
        if parent_tag:
            matching_nodes = parent_tag.find_all(string=re.compile(keyword))
        return matching_nodes

    def find(self, tag_desc: dict, reg: re = None) -> str:
        """
        依据容器的定位描述查找节点文本

        Args:
            tag_desc (:dict): 容器的定位描述，采用如下键值对格式：
                                {
                                    'name':'div', # 标签的名称，可为字符串或正则表达式。\n
                                    'id':'noticeArea', # 标签的ID\n
                                    'attrs':{‘class’: ‘test’} # 标签的属性，字典类型\n
                                }
            reg (re): 匹配内容的正则表达式

        Returns:
            str: 提取的关键词内容
        """
        tag_name = None
        tag_id = None
        tag_attrs = None

        if tag_desc and 'name' in tag_desc and tag_desc['name'] != '':
            tag_name = tag_desc['name']
        if tag_desc and 'id' in tag_desc and tag_desc['id'] != '':
            tag_id = tag_desc['id']
        if tag_desc and 'attrs' in tag_desc and tag_desc['attrs'] != '':
            tag_attrs = tag_desc['attrs']
        if tag_id:
            if tag_attrs:
                tag = self.soup.find(name=tag_name, id=tag_id, attrs=tag_attrs)
            else:
                tag = self.soup.find(name=tag_name, id=tag_id)
        else:
            tag = self.soup.find(name=tag_name, attrs=tag_attrs)

        if tag:
            if reg:
                match = re.search(reg, tag.text)
                if match: return match.group(1)
                else: return None
            else: return tag.text
        else: return None

    def find_keyword_content(self,
                             keyword: str,
                             reg: re,
                             parent_desc: dict = None) -> str:
        """
        使用BeautifulSoup提取关键词内容

        Args:
            keyword (str): 关键词
            reg (re): 匹配内容的正则表达式
            parent_desc (:dict): 父容器的定位描述，采用如下键值对格式：
                                {
                                    'name':'div', # 标签的名称，可为字符串或正则表达式。\n
                                    'id':'noticeArea', # 标签的ID\n
                                    'attrs':{‘class’: ‘test’} # 标签的属性，字典类型\n
                                }

        Returns:
            str: 提取的关键词内容
        """
        keyword_content = ""
        nodes = self.locate(keyword, parent_desc)
        if nodes:
            for element in nodes:
                match = re.search(reg, element)
                if match:
                    keyword_content = match.group(1)
                    break
        return keyword_content

    def extract_table(self, header_map: dict, parent_desc: dict = None) -> []:
        """
        使用BeautifulSoup提取表格内容

        Args:
            header_map (dict): 表头名称映射的属性字典
            parent_desc (:dict): 父容器的定位描述，采用如下键值对格式：
                                {
                                    'name':'div', # 标签的名称，可为字符串或正则表达式。\n
                                    'id':'noticeArea', # 标签的ID\n
                                    'attrs':{‘class’: ‘test’} # 标签的属性，字典类型\n
                                }

        Returns:
            []: 提取的关键词内容
        """
        box = self.locate_box(parent_desc)
        if not box:
            return []
        table = box.find('table')
        if not table:
            return []
        rows = table.find_all('tr')
        header_row = rows[0]
        data_rows = rows[1:]

        headers = [header.text.strip() for header in header_row.find_all('th')]
        datas = []
        for row in data_rows:
            cells = row.find_all('td')
            values = [cell.text.strip() for cell in cells]
            data = dict(zip(headers, values))
            datas.append(data)
        # 使用表头映射字典header_map转换列表
        result = [{
            new_key: item.get(old_key)
            for new_key, old_key in header_map.items()
        } for item in datas]
        # 循环的写法：
        # for row in datas:
        #     new_row = {}
        #     for key in header_map.keys():
        #         for rkey in row.keys():
        #             if header_map[key] == rkey:
        #                 new_row[key] = row[rkey]
        #                 break
        #     result.append(new_row)
        return result


# 测试代码：
if __name__ == '__main__':
    # 测试提取函数
    html_content = '''
    <body>
        <div class="clearfix" id="newsText">
            <div>
                <div class="protect" id="noticeArea">
                    <meta http-equiv="content-type" content="text/html;charset=utf-8">
                    <p data-v-6b13d514=""></p>
                    <h4><strong>一、项目编号：N5100012022001470</strong>asdfadsaf</h4>
                    <h4><strong>二、项目名称：西南区域环境空气质量预测预报中心能力提升项目</strong></h4>
                    <h4><strong>三、采购结果</strong></h4>
                    <div class="noticeBidResult-noticeBidResult _notice_content_noticeBidResult-noticeBidResult supplier dynamic-form-editor"
                        id="_notice_content_noticeBidResult-noticeBidResult">
                        <div>
                            <p class="u-content">合同包1:</p>
                            <table width="100%" border="1" cellspacing="0">
                                <thead>
                                    <tr style="height:36.4pt">
                                        <th>供应商名称</th>
                                        <th>供应商地址</th>
                                        <th>中标（成交）金额</th>
                                    </tr>
                                </thead>
                                <tbody>
                                    <tr>
                                        <td style="width:400px;word-break:break-all;">四川省国土空间生态修复与地质灾害防治研究院</td>
                                        <td>成都市人民北路一段25号</td>
                                        <td class="alignright"
                                            style="width:200px;text-align: right;white-space:nowrap;">
                                            <span>22,390,000.00元</span>
                                        </td>
                                    </tr>
                                </tbody>
                            </table>
                        </div>
                        <div>
                            <p class="u-content">合同包2:</p>
                            <table width="100%" border="1" cellspacing="0">
                                <thead>
                                    <tr style="height:36.4pt">
                                        <th>供应商名称</th>
                                        <th>供应商地址</th>
                                        <th>中标（成交）金额</th>
                                    </tr>
                                </thead>
                                <tbody>
                                    <tr>
                                        <td style="width:400px;word-break:break-all;">北京东方至远科技股份有限公司</td>
                                        <td>北京市海淀区花园北路14号66幢2层207号</td>
                                        <td class="alignright"
                                            style="width:200px;text-align: right;white-space:nowrap;">
                                            <span>2,692,128.00元</span>
                                        </td>
                                    </tr>
                                </tbody>
                            </table>
                        </div>
                        <div>
                            <p class="u-content">合同包3:</p>
                            <table width="100%" border="1" cellspacing="0">
                                <thead>
                                    <tr style="height:36.4pt">
                                        <th>供应商名称</th>
                                        <th>供应商地址</th>
                                        <th>中标（成交）金额</th>
                                    </tr>
                                </thead>
                                <tbody>
                                    <tr>
                                        <td style="width:400px;word-break:break-all;">成都理工大学</td>
                                        <td>成都市二仙桥东三路1号</td>
                                        <td class="alignright"
                                            style="width:200px;text-align: right;white-space:nowrap;">
                                            <span>1,170,200.00元</span>
                                        </td>
                                    </tr>
                                </tbody>
                            </table>
                        </div>
                    </div>
                    <h4><strong>九、凡对本次公告内容提出询问，请按以下方式联系。</h4></strong>
                    <div class="innercontent">
                        <h6>1.采购人信息</h6>
                        <p>名称：<span
                                class="u-content noticePurchase-purchaserOrgName _notice_content_noticePurchase-purchaserOrgName dynamic-form-editor"
                                id="_notice_content_noticePurchase-purchaserOrgName">四川省自然资源厅</span>
                        </p>
                        <p>地址：<span
                                class="u-content noticePurchase-purchaserOrgAddress _notice_content_noticePurchase-purchaserOrgAddress dynamic-form-editor"
                                id="_notice_content_noticePurchase-purchaserOrgAddress">成都市青羊区百卉路4号</span>
                        </p>
                        <p>联系方式：<span
                                class="u-content noticePurchase-purchaserLinkTel _notice_content_noticePurchase-purchaserLinkTel dynamic-form-editor"
                                id="_notice_content_noticePurchase-purchaserLinkTel">028-87036212</span>
                        </p>
                    </div>
                </div>
            </div>
        </div>
    </body>
    '''
    #测试find接口：
    # soup = BeautifulSoup(html_content, 'html.parser')
    # f = soup.find(attrs={'class': 'protect'})
    # f1 = f.find(name='p')
    # f2 = f.find(string=re.compile('项目编号'))
    # print("find:", f)
    # print("sub find f1:", f1)
    # print("sub find f2:", f2)

    analyzer = HTMLAnalyzer()
    # 需要加载的网页链接
    analyzer.load_html(html_content)
    # analyzer.load_page(
    #     'https://ggzyjy.sc.gov.cn/cyzdxm/019002/019002002/20221110/8a69ce88845d0ce301846087352e087eCY.html'
    #     # 'https://ggzyjy.sc.gov.cn/cyzdxm/019002/019002002/20221221/1181671806399197184_1.html'
    #     # 'https://ggzyjy.sc.gov.cn/jyxx/002002/002002003/20221116/8a69c5e5845d0d1401847ebcb1977ae1.html'
    # )
    # 测试提取表格：
    header_map = {
        'cunit': '供应商名称',
        'cunitAddress': '供应商地址',
        'amount': '中标（成交）金额'
    }
    items = analyzer.extract_table(header_map,
                                   parent_desc={
                                       'name': 'div',
                                       'attrs': {
                                           'class':
                                           'noticeBidResult-noticeBidResult'
                                       }
                                   })
    print('提取的表格数据：')
    print(items)
    # 测试关键词内容匹配：
    nodes = analyzer.locate('项目编号')
    print('只使用关键词搜索结果：', nodes)
    nodes = analyzer.locate(keyword='项目编号', parent_desc={'name': 'div'})
    print('使用关键词+标签搜索结果：', nodes)
    print('测试xpath接口：')
    project_number_xpath = analyzer.xextract_keyword_content(
        "项目编号", '//div[@class="protect" and @id="noticeArea"]',
        r'项目编号：([A-Za-z0-9]+)')
    print("project_number:", project_number_xpath)

    project_name_xpath = analyzer.xextract_keyword_content(
        "项目名称", '//div[@class="protect" and @id="noticeArea"]', r'项目名称：(.+)')
    print("project_name:", project_name_xpath)

    print('测试bs4接口：')
    project_number_bs4 = analyzer.find({'name': 'h4'})
    print("get project_number by find:", project_number_bs4)
    project_number_bs4 = analyzer.find_keyword_content("项目编号",
                                                       r'项目编号：([A-Za-z0-9]+)')
    print("get project_number only by keyword:", project_number_bs4)
    project_number_bs4 = analyzer.find_keyword_content(
        "项目编号", r'项目编号：([A-Za-z0-9]+)', parent_desc={'name': 'h4'})
    print("get project_number by tag:", project_number_bs4)
    project_number_bs4 = analyzer.find_keyword_content(
        "项目编号",
        r'项目编号：([A-Za-z0-9]+)',
        parent_desc={'attrs': {
            'class': 'protect'
        }})
    print("get project_number by class:", project_number_bs4)
    project_number_bs4 = analyzer.find_keyword_content(
        "项目编号", r'项目编号：([A-Za-z0-9]+)', parent_desc={'id': 'noticeArea'})
    print("get project_number by id:", project_number_bs4)

    # 注意：这里不能使用h4,因为h4匹配的时候只能定位到项目编号，应使用更高一级的父容器：
    project_name_bs4 = analyzer.find_keyword_content(
        "项目名称", r'项目名称：(.+)', parent_desc={'name': 'div'})
    print("get project_name by tag:", project_name_bs4)
    project_name_bs4 = analyzer.find_keyword_content(
        "项目名称", r'项目名称：(.+)', parent_desc={'attrs': {
            'class': 'protect'
        }})
    print("get project_name by class:", project_name_bs4)
    project_name_bs4 = analyzer.find_keyword_content(
        "项目名称", r'项目名称：(.+)', parent_desc={'id': 'noticeArea'})
    print("get project_name by id:", project_name_bs4)
    # 提取采购人信息
    purchaser_name = ""
    purchaser_name = analyzer.find({
        'name': 'span',
        'attrs': {
            'class': 'noticePurchase-purchaserOrgName'
        }
    })
    purchaser_address = analyzer.find({
        'name': 'span',
        'attrs': {
            'class': 'noticePurchase-purchaserOrgAddress'
        }
    })
    purchaser_contact = analyzer.find(
        {
            'name': 'span',
            'attrs': {
                'class': 'noticePurchase-purchaserLinkTel'
            }
        }, "")
    purchaser_contact2 = analyzer.find(
        {
            'name': 'span',
            'attrs': {
                'class': 'noticePurchase-purchaserLinkTel'
            }
        }, r"(?:\d{3}-)?(\d{8})")
    print('采购人信息:')
    print('名称:', purchaser_name)
    print('地址:', purchaser_address)
    print('联系方式:', purchaser_contact)
    print('联系方式(没有区号的):', purchaser_contact2)
# else:
#     print("Failed to load page")
